from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import matplotlib
import os
import pylab
import random
import sys
import time
from sets import Set
from scipy import stats
import numpy as np

def compute_histogram_against_document_distribution_files():
    inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_debugPercentage_weight50_completed"
    outputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_debugPercentage_weight50_histogram"

    inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_1Percentage_weight50_completed"
    outputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_1Percentage_weight50_histogram"
    
    inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_3Percentage_weight50_completed"
    outputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_3Percentage_weight50_histogram"
    
    inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_5Percentage_weight50_completed"
    outputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_5Percentage_weight50_histogram"
    
    inputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_10Percentage_weight50_completed"
    outputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_10Percentage_weight50_histogram"
    
    inputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_15Percentage_weight50_completed"
    outputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_15Percentage_weight50_histogram"
    
    inputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_20Percentage_weight50_completed"
    outputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_20Percentage_weight50_histogram"
    
    inputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_30Percentage_weight50_completed"
    outputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_30Percentage_weight50_histogram"
    
    inputFileName9 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_40Percentage_weight50_completed"
    outputFileName9 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_40Percentage_weight50_histogram"
    
    inputFileName10 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_50Percentage_weight50_completed"
    outputFileName10 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_50Percentage_weight50_histogram"
    
    inputFileName11 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_60Percentage_weight50_completed"
    outputFileName11 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_60Percentage_weight50_histogram"
    
    inputFileName12 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_70Percentage_weight50_completed"
    outputFileName12 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_70Percentage_weight50_histogram"
    
    inputFileName13 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_80Percentage_weight50_completed"
    outputFileName13 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_80Percentage_weight50_histogram"
    
    inputFileName14 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_90Percentage_weight50_completed"
    outputFileName14 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_90Percentage_weight50_histogram"

    processingFileTuplelist = []
    processingFileTuplelist.append( (inputFileName1,outputFileName1) )
    processingFileTuplelist.append( (inputFileName2,outputFileName2) )
    processingFileTuplelist.append( (inputFileName3,outputFileName3) )
    processingFileTuplelist.append( (inputFileName4,outputFileName4) )
    processingFileTuplelist.append( (inputFileName5,outputFileName5) )
    processingFileTuplelist.append( (inputFileName6,outputFileName6) )
    processingFileTuplelist.append( (inputFileName7,outputFileName7) )
    processingFileTuplelist.append( (inputFileName8,outputFileName8) )
    processingFileTuplelist.append( (inputFileName9,outputFileName9) )
    processingFileTuplelist.append( (inputFileName10,outputFileName10) )
    processingFileTuplelist.append( (inputFileName11,outputFileName11) )
    processingFileTuplelist.append( (inputFileName12,outputFileName12) )
    processingFileTuplelist.append( (inputFileName13,outputFileName13) )
    processingFileTuplelist.append( (inputFileName14,outputFileName14) )

    for processingFileTuple in processingFileTuplelist:
        # key: the rounding value
        # value: freq
        freqOfFreqDict = {}
        (currentInputFileName,currentOutputFileName) = processingFileTuple
        inputFileHandler = open(currentInputFileName,"r")
        outputFileHandler = open(currentOutputFileName,"w")
        
        currentLine = inputFileHandler.readline()
        while currentLine:
            currentLineElements = currentLine.strip().split(" ")
            percentageValue = float(currentLineElements[3]) * 100
            roundIntValue = round(percentageValue)
            if roundIntValue not in freqOfFreqDict:
                freqOfFreqDict[roundIntValue] = 1
            else:
                freqOfFreqDict[roundIntValue] += 1
            currentLine = inputFileHandler.readline()
        inputFileHandler.close()
        keyList = []
        keyList = freqOfFreqDict.keys()
        for currentKey in keyList:
            # print currentKey,freqOfFreqDict[currentKey]
            outputLine = str(currentKey) + " " + str(freqOfFreqDict[currentKey]) + "\n"
            outputFileHandler.write(outputLine)
                
        inputFileHandler.close()
        outputFileHandler.close()
   
        print "OVERALL:"
        print "currentInputFileName:",currentInputFileName
        print "currentOutputFileName:",currentOutputFileName
        
def compute_complete_document_distribution_files():
    # DocumentDistribution_debugPercentage_weight50
    # DocumentDistribution_1Percentage_weight50
    # DocumentDistribution_3Percentage_weight50
    # DocumentDistribution_5Percentage_weight50
    # DocumentDistribution_10Percentage_weight50
    # DocumentDistribution_15Percentage_weight50
    # DocumentDistribution_20Percentage_weight50
    # DocumentDistribution_30Percentage_weight50
    # DocumentDistribution_40Percentage_weight50
    # DocumentDistribution_50Percentage_weight50
    # DocumentDistribution_60Percentage_weight50
    # DocumentDistribution_70Percentage_weight50
    # DocumentDistribution_80Percentage_weight50
    # DocumentDistribution_90Percentage_weight50
    # DocumentDistribution_100Percentage_weight50
    docIDWithNumOfPostingsDict = {}
    inputFileName0 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sorted_by_XdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument_sortedByTrecID"
    inputFileHandler = open(inputFileName0,"r")
    currentLine = inputFileHandler.readline()
    sumTotalNumOfUniquePostings = 0
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[1]
        currentNumOfUniquePostings = int(currentLineElements[2])
        sumTotalNumOfUniquePostings += currentNumOfUniquePostings
        if currentDocID not in docIDWithNumOfPostingsDict:
            docIDWithNumOfPostingsDict[currentDocID] = currentNumOfUniquePostings
        else:
            exit(1)
        currentLine = inputFileHandler.readline()
    print "len(docIDWithNumOfPostingsDict):",len(docIDWithNumOfPostingsDict)
    print "docIDWithNumOfPostingsDict['10']:",docIDWithNumOfPostingsDict['10']
    print "docIDWithNumOfPostingsDict['10']:",docIDWithNumOfPostingsDict['10']
    print "sumTotalNumOfUniquePostings:",sumTotalNumOfUniquePostings
    
    inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_debugPercentage_weight50"
    inputFileHandler = open(inputFileName1,"r")
    
    outputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_debugPercentage_weight50_completed"
    outputFileHandler = open(outputFileName1,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_1Percentage_weight50"
    inputFileHandler = open(inputFileName2,"r")
    
    outputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_1Percentage_weight50_completed"
    outputFileHandler = open(outputFileName2,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_3Percentage_weight50"
    inputFileHandler = open(inputFileName3,"r")
    
    outputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_3Percentage_weight50_completed"
    outputFileHandler = open(outputFileName3,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_5Percentage_weight50"
    inputFileHandler = open(inputFileName4,"r")
    
    outputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_5Percentage_weight50_completed"
    outputFileHandler = open(outputFileName4,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_10Percentage_weight50"
    inputFileHandler = open(inputFileName5,"r")
    
    outputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_10Percentage_weight50_completed"
    outputFileHandler = open(outputFileName5,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_15Percentage_weight50"
    inputFileHandler = open(inputFileName6,"r")
    
    outputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_15Percentage_weight50_completed"
    outputFileHandler = open(outputFileName6,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_20Percentage_weight50"
    inputFileHandler = open(inputFileName7,"r")
    
    outputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_20Percentage_weight50_completed"
    outputFileHandler = open(outputFileName7,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_30Percentage_weight50"
    inputFileHandler = open(inputFileName8,"r")
    
    outputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_30Percentage_weight50_completed"
    outputFileHandler = open(outputFileName8,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName9 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_40Percentage_weight50"
    inputFileHandler = open(inputFileName9,"r")
    
    outputFileName9 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_40Percentage_weight50_completed"
    outputFileHandler = open(outputFileName9,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName10 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_50Percentage_weight50"
    inputFileHandler = open(inputFileName10,"r")
    
    outputFileName10 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_50Percentage_weight50_completed"
    outputFileHandler = open(outputFileName10,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName11 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_60Percentage_weight50"
    inputFileHandler = open(inputFileName11,"r")
    
    outputFileName11 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_60Percentage_weight50_completed"
    outputFileHandler = open(outputFileName11,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName12 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_70Percentage_weight50"
    inputFileHandler = open(inputFileName12,"r")
    
    outputFileName12 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_70Percentage_weight50_completed"
    outputFileHandler = open(outputFileName12,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName13 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_80Percentage_weight50"
    inputFileHandler = open(inputFileName13,"r")
    
    outputFileName13 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_80Percentage_weight50_completed"
    outputFileHandler = open(outputFileName13,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    inputFileName14 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_90Percentage_weight50"
    inputFileHandler = open(inputFileName14,"r")
    
    outputFileName14 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_90Percentage_weight50_completed"
    outputFileHandler = open(outputFileName14,"w")
    
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = currentLineElements[0]
        currentNumOfPostingsCollected = int(currentLineElements[1])
        outputLine = str(currentDocID) + " " + str(currentNumOfPostingsCollected) + " " + str(docIDWithNumOfPostingsDict[currentDocID]) + " " + str(currentNumOfPostingsCollected/docIDWithNumOfPostingsDict[currentDocID]) + "\n"
        outputFileHandler.write(outputLine)
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    
    print "OVERALL:"
    print "inputFileName0:",inputFileName0
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2
    print "inputFileName3:",inputFileName3
    print "inputFileName4:",inputFileName4
    print "inputFileName5:",inputFileName5
    print "inputFileName6:",inputFileName6
    print "inputFileName7:",inputFileName7
    print "inputFileName8:",inputFileName8
    print "inputFileName9:",inputFileName9
    print "inputFileName10:",inputFileName10
    print "inputFileName11:",inputFileName11
    print "inputFileName12:",inputFileName12
    print "inputFileName13:",inputFileName13
    print "inputFileName14:",inputFileName14
    print "outputFileName1:",outputFileName1
    print "outputFileName2:",outputFileName2
    print "outputFileName3:",outputFileName3
    print "outputFileName4:",outputFileName4
    print "outputFileName5:",outputFileName5
    print "outputFileName6:",outputFileName6
    print "outputFileName7:",outputFileName7
    print "outputFileName8:",outputFileName8
    print "outputFileName9:",outputFileName9
    print "outputFileName10:",outputFileName10
    print "outputFileName11:",outputFileName11
    print "outputFileName12:",outputFileName12
    print "outputFileName13:",outputFileName13
    print "outputFileName14:",outputFileName14

def compute_complete_histogram():
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_debugPercentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_1Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_3Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_5Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_10Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_15Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_20Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_30Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_40Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_50Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_60Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_70Percentage_weight50_histogram"
    # inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_80Percentage_weight50_histogram"
    inputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_90Percentage_weight50_histogram"
    inputFileHandler = open(inputFileName,"r")
    sumOfDoc = 0
    for line in inputFileHandler.readlines():
        sumOfDoc += int(line.strip().split(" ")[1])
    print "sumOfDoc:",sumOfDoc
    inputFileHandler.close()
        
    outputFileName = inputFileName + "_complete"
    outputFileHanlder = open(outputFileName,"w")
    
    inputFileHandler = open(inputFileName,"r")
    
    for line in inputFileHandler.readlines():
        numerator = int(line.strip().split(" ")[1])
        outputLine = line.strip() + " " + str(numerator/sumOfDoc) + "\n"
        outputFileHanlder.write(outputLine)
    
    print "OVERALL:"
    print "inputFileName:",inputFileName
    print "outputFileName:",outputFileName
    
    inputFileHandler.close()
    outputFileHanlder.close()
    
    
    
def compute_raw_document_distribution_files():
    docIDDict = {}
    TOTAL_NUM_OF_POSTINGS = 6451948010
    numOfPostingNeededToBePoppedAtDebugPercentage = 5000000
    numOfPostingNeededToBePoppedAt1Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.01)
    numOfPostingNeededToBePoppedAt3Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.03)
    numOfPostingNeededToBePoppedAt5Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.05)
    numOfPostingNeededToBePoppedAt10Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.1)
    numOfPostingNeededToBePoppedAt15Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.15)
    numOfPostingNeededToBePoppedAt20Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.2)
    numOfPostingNeededToBePoppedAt30Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.3)
    numOfPostingNeededToBePoppedAt40Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.4)
    numOfPostingNeededToBePoppedAt50Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.5)
    numOfPostingNeededToBePoppedAt60Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.6)
    numOfPostingNeededToBePoppedAt70Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.7)
    numOfPostingNeededToBePoppedAt80Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.8)
    numOfPostingNeededToBePoppedAt90Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.9)
    numOfPostingNeededToBePoppedAt100Percentage = int(TOTAL_NUM_OF_POSTINGS * 1.0)
    print "numOfPostingNeededToBePoppedAtDebugPercentage:",numOfPostingNeededToBePoppedAtDebugPercentage
    print "numOfPostingNeededToBePoppedAt1Percentage:",numOfPostingNeededToBePoppedAt1Percentage
    print "numOfPostingNeededToBePoppedAt3Percentage:",numOfPostingNeededToBePoppedAt3Percentage
    print "numOfPostingNeededToBePoppedAt5Percentage:",numOfPostingNeededToBePoppedAt5Percentage
    print "numOfPostingNeededToBePoppedAt10Percentage:",numOfPostingNeededToBePoppedAt10Percentage
    print "numOfPostingNeededToBePoppedAt15Percentage:",numOfPostingNeededToBePoppedAt15Percentage
    print "numOfPostingNeededToBePoppedAt20Percentage:",numOfPostingNeededToBePoppedAt20Percentage
    print "numOfPostingNeededToBePoppedAt30Percentage:",numOfPostingNeededToBePoppedAt30Percentage
    print "numOfPostingNeededToBePoppedAt40Percentage:",numOfPostingNeededToBePoppedAt40Percentage
    print "numOfPostingNeededToBePoppedAt50Percentage:",numOfPostingNeededToBePoppedAt50Percentage
    print "numOfPostingNeededToBePoppedAt60Percentage:",numOfPostingNeededToBePoppedAt60Percentage
    print "numOfPostingNeededToBePoppedAt70Percentage:",numOfPostingNeededToBePoppedAt70Percentage
    print "numOfPostingNeededToBePoppedAt80Percentage:",numOfPostingNeededToBePoppedAt80Percentage
    print "numOfPostingNeededToBePoppedAt90Percentage:",numOfPostingNeededToBePoppedAt90Percentage
    print "numOfPostingNeededToBePoppedAt100Percentage:",numOfPostingNeededToBePoppedAt100Percentage
    numOfBytes = 0
    numOfPostingPopped = 0
    numOfPostingBeInTOP10 = 0
    
    inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/allPostingsBeingPopped20140320Afternoon_weight_50_WHOLE"
    inputFileHandler = open(inputFileName3,"rb")
    print "--->posting file to evaluate:",inputFileName3
    statinfo = os.stat(inputFileName3)
    fileSize = statinfo.st_size
    print "file size:",fileSize
    numOfResultsReturnedCountedUpTo10 = 0
    
    while numOfBytes < fileSize:
        # old version
        # each time, just read the info of ONE posting, too few
        # byteString = inputFileHandler.read(4 + 4 + 4)
        # (termID,docID,currentProbability) = unpack( "2I1f", byteString)
        
        # current version
        # each time, read the info of 1M postings
        byteStringBuffer = inputFileHandler.read( 1000000 * 16)
        byteStringBufferIndexPosition = 0
        for i in range(0,1000000):
            byteString = byteStringBuffer[byteStringBufferIndexPosition:byteStringBufferIndexPosition+16]
            byteStringBufferIndexPosition += 16
            (termID,docID,currentProbability,impactScore) = unpack( "2I2f", byteString)
            if docID not in docIDDict:
                docIDDict[docID] = 1
            else:
                docIDDict[docID] += 1
    
            if numOfPostingPopped == numOfPostingNeededToBePoppedAtDebugPercentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt1Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt3Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt5Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt10Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt15Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt20Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt30Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt40Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt50Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt60Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt70Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt80Percentage or numOfPostingPopped == numOfPostingNeededToBePoppedAt90Percentage:
                if numOfPostingPopped == numOfPostingNeededToBePoppedAtDebugPercentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_debugPercentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt1Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_1Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt3Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_3Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt5Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_5Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt10Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_10Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt15Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_15Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt20Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_20Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt30Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_30Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt40Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_40Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt50Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_50Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
    
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt60Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_60Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt70Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_70Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
    
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt80Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_80Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt90Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_90Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
    
                if numOfPostingPopped == numOfPostingNeededToBePoppedAt100Percentage:
                    outputFileName = "/home/diaosi/workspace/web-search-engine-wei-2014-March/results/DocumentDistribution_100Percentage_weight50"
                    outputFileHandler = open(outputFileName,"w")
                    docIDList = []
                    docIDList = docIDDict.keys()
                    docIDList.sort(cmp=None, key=None, reverse=False)
                    for currentDOCID in docIDList:
                        outputLine = str(currentDOCID) + " " + str(docIDDict[currentDOCID]) + "\n"
                        outputFileHandler.write(outputLine)
                    outputFileHandler.close()
                    print "len(docIDDict):",len(docIDDict)
                    print "numOfPostingPopped:",numOfPostingPopped
                    print "outputFileName:",outputFileName
                
    
            numOfBytes += 12
            numOfPostingPopped += 1
            if numOfPostingPopped % 1000000 == 0:
                print str(numOfPostingPopped),"postings have been examined."
    
    inputFileHandler.close()

print "Program Begins..."
compute_raw_document_distribution_files
# compute_complete_document_distribution_files()
# compute_histogram_against_document_distribution_files()
# compute_complete_histogram()
print "Program Ends."


